package cn.lh.spider;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class CookieSpider implements PageProcessor {
	
	private Site site = Site.me()
//			.addCookie("git.oschina.net", "remember_user_token", "6BkVU--d80efbe7a8383cba000577ee19a9d6fa24649dc8")
//			.addCookie("Hm_lvt_a411c4d1664dd70048ee98afe7b28f0b", "1484877352")
//			.addCookie("_user_behavior_", "4d1a7116-511a-4c81-afd2-62ed496bf91f")
//			.addCookie("aliyungf_tc", "AQAAAG6D7wwMAggAUkha3p2z5RCD+KWT")
			.addCookie("git_osc_session", "BAh7CUkiD3Nlc3Npb25faWQGOgZFVEkiJTRlNTZiYmFjNGNhMGU3ZjkyNmMyNjA2YjI5ZTMxOGI0BjsAVEkiDGNhcHRjaGEGOwBGSSILUE5HTjZUBjsAVEkiGXdhcmRlbi51c2VyLnVzZXIua2V5BjsAVFsHWwZpA215EEkiIiQyYSQxMCRLaDhMMU55V1BmYU9zTlUuMmo1SjBPBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMVhFYVk3clJRY1BLRngzT3RhZG9tVnVJODhDRk44bEYxOUxzdXl5bXFaMWc9BjsARg%3D%3D--ca962c6bf0bcd3c44b6ad1562aa3e232399e7c4d")
//			.addCookie("oschina_new_user", "false")
//			.addCookie("remote_way", "http")
//			.addCookie("tz", "Asia%2FShanghai")
//			.addCookie("user_locale", "zh-CN")
			.setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);
	public void process(Page page) {
		page.putField("html", page.getHtml());
		page.putField("title", page.getHtml().xpath("//a[@title='com.lh.jeecg']/html()"));
	}

	public Site getSite() {
		return site;
	}
	public static void main(String[] args) {
		Spider.create(new CookieSpider())
		.addUrl("https://git.oschina.net") 
		//.addPipeline(new FilePipeline("d://a"))
        .run();
	}

}
